From: Joseph Marrero Corchado Date: Tue, 7 Apr 2026 19:24:45 +0000 (-0400) Subject: generator: Fix soft-reboot for var, sysroot, and boot X-Git-Tag: archive/raspbian/2026.2-1+rpi1~1^2~10^2^2~1^2 X-Git-Url: https://dgit.raspbian.org/%22http://www.example.com/cgi/%22/%22http:/www.example.com/cgi/%22?a=commitdiff_plain;h=a88e66eaf86a3c7b0060073757c02245196f759d;p=ostree.git generator: Fix soft-reboot for var, sysroot, and boot A bare `systemctl soft-reboot` on ostree/bootc systems was broken in several ways because the generator and prepare-root were not accounting for the fact that soft-reboot does not re-run the initramfs. The var.mount unit had DefaultDependencies=yes, which pulled in implicit After= dependencies on device units. After soft-reboot, these device units get stuck in 'tentative' state while udev restarts, causing var.mount to stall indefinitely. Fix this by setting DefaultDependencies=no with explicit ordering After=local-fs-pre.target sysroot.mount. For /sysroot, systemd auto-generates the mount unit from mountinfo with Conflicts=umount.target, causing it to be unmounted during soft-reboot shutdown. Generate a drop-in with DefaultDependencies=no to prevent this. We use a drop-in because the generator does not know the What= device parameter — systemd gets that from mountinfo. For /boot on same-partition setups, move the bind-mount from ostree-prepare-root into the generator as a full boot.mount unit with DefaultDependencies=no. This handles normal boot, bare soft-reboot, and staged deployment soft-reboot uniformly. The static (non-systemd) path in ostree-prepare-root-static.c retains its own bind-mount since the generator does not run there. Validated with plain disk and RAID1 kola tests on FCOS 43. Fixes: https://issues.redhat.com/browse/RHEL-154075 Assisted-by: OpenCode (Claude Opus 4.6) Signed-off-by: Joseph Marrero Corchado --- diff --git a/src/libostree/ostree-impl-system-generator.c b/src/libostree/ostree-impl-system-generator.c index f0116251..a21ff6bb 100644 --- a/src/libostree/ostree-impl-system-generator.c +++ b/src/libostree/ostree-impl-system-generator.c @@ -150,6 +150,177 @@ _ostree_sysroot_parse_bootlink_aboot (const char *bootlink, char **out_osname, G return TRUE; } +/* Generate a drop-in for a mount unit to set DefaultDependencies=no. + * + * By default, systemd auto-generates mount units from /proc/self/mountinfo with + * DefaultDependencies=yes, which includes Conflicts=umount.target. This causes + * the mount to be unmounted during soft-reboot shutdown. Since soft-reboot doesn't + * re-run the initramfs, these mounts are never remounted, breaking bootc/ostree. + * + * By generating a drop-in, we override just the DefaultDependencies setting + * while letting systemd handle the actual mount parameters (What=, etc.) from + * the existing mount in /proc/self/mountinfo. + * + * @mount_unit: The mount unit name (e.g., "sysroot.mount", "boot.mount") + * @mount_point: The mount point path for the comment (e.g., "/sysroot", "/boot") + */ +static gboolean +generate_mount_unit_dropin (int normal_dir_dfd, const char *mount_unit, const char *mount_point, + GError **error) +{ + GCancellable *cancellable = NULL; + + /* Create the drop-in directory (e.g., sysroot.mount.d) */ + g_autofree char *dropin_dir = g_strdup_printf ("%s.d", mount_unit); + if (!glnx_shutil_mkdir_p_at (normal_dir_dfd, dropin_dir, 0755, cancellable, error)) + return FALSE; + + g_auto (GLnxTmpfile) tmpf = { + 0, + }; + if (!glnx_open_tmpfile_linkable_at (normal_dir_dfd, ".", O_WRONLY | O_CLOEXEC, &tmpf, error)) + return FALSE; + g_autoptr (GOutputStream) outstream = g_unix_output_stream_new (tmpf.fd, FALSE); + gsize bytes_written; + + /* Generate drop-in to set DefaultDependencies=no. + * + * Key points: + * - DefaultDependencies=no prevents Conflicts=umount.target from being added + * - This allows the mount to survive soft-reboot + * - The actual mount (What=, Where=, etc.) comes from /proc/self/mountinfo + */ + if (!g_output_stream_printf (outstream, &bytes_written, cancellable, error, + "##\n# Automatically generated by ostree-system-generator\n" + "# Preserve %s across soft-reboot\n##\n\n" + "[Unit]\n" + "DefaultDependencies=no\n" + "After=local-fs-pre.target\n" + "Before=local-fs.target\n", + mount_point)) + return FALSE; + if (!g_output_stream_flush (outstream, cancellable, error)) + return FALSE; + g_clear_object (&outstream); + if (!glnx_fchmod (tmpf.fd, 0644, error)) + return FALSE; + + g_autofree char *dropin_path = g_strdup_printf ("%s/ostree-softreboot.conf", dropin_dir); + if (!glnx_link_tmpfile_at (&tmpf, GLNX_LINK_TMPFILE_NOREPLACE, normal_dir_dfd, dropin_path, + error)) + return FALSE; + + return TRUE; +} + +/* Generate a drop-in for sysroot.mount to preserve it across soft-reboot. + * + * /sysroot is special: it's mounted in the initramfs and can never be + * remounted without re-running the initramfs. Since soft-reboot skips + * the initramfs, we must prevent systemd from unmounting it. + */ +static gboolean +sysroot_mount_generator (const char *normal_dir, GError **error) +{ + glnx_autofd int normal_dir_dfd = -1; + + if (!glnx_opendirat (AT_FDCWD, normal_dir, TRUE, &normal_dir_dfd, error)) + return FALSE; + + if (!generate_mount_unit_dropin (normal_dir_dfd, "sysroot.mount", "/sysroot", error)) + return FALSE; + + return TRUE; +} + +/* Generate boot.mount for /boot when it's on the same partition as /sysroot. + * + * When /boot is on a separate partition, systemd auto-generates a mount unit + * from /proc/self/mountinfo and will remount it after soft-reboot. No action + * needed in that case. + * + * When /boot is on the same partition (detected by /sysroot/boot/loader being + * a symlink), a bind mount from /sysroot/boot is needed. Previously this was + * done in the initramfs (ostree-prepare-root), but that breaks on bare + * soft-reboot since the initramfs doesn't re-run. By generating boot.mount + * here, the generator handles all three boot scenarios: + * 1. Normal boot (generator runs from initramfs) + * 2. Bare soft-reboot (generator re-runs) + * 3. Staged deployment soft-reboot (generator re-runs for new root) + * + * See: https://github.com/ostreedev/ostree/pull/3487 + * https://github.com/ostreedev/ostree/pull/3571 + */ +static gboolean +boot_mount_generator (const char *normal_dir, GError **error) +{ + GCancellable *cancellable = NULL; + static const char boot_path[] = "/boot"; + struct stat stbuf; + + /* Check if /boot is on the same partition as /sysroot by looking for + * /sysroot/boot/loader as a symlink. This is the same check used by + * otcore_mount_boot() in the initramfs path. + */ + if (!(lstat ("/sysroot/boot/loader", &stbuf) == 0 && S_ISLNK (stbuf.st_mode))) + return TRUE; /* /boot is a separate partition, systemd handles it */ + + /* Verify the target /boot directory exists */ + if (!(lstat ("/boot", &stbuf) == 0 && S_ISDIR (stbuf.st_mode))) + return TRUE; + + glnx_autofd int normal_dir_dfd = -1; + if (!glnx_opendirat (AT_FDCWD, normal_dir, TRUE, &normal_dir_dfd, error)) + return FALSE; + + g_auto (GLnxTmpfile) tmpf = { + 0, + }; + if (!glnx_open_tmpfile_linkable_at (normal_dir_dfd, ".", O_WRONLY | O_CLOEXEC, &tmpf, error)) + return FALSE; + g_autoptr (GOutputStream) outstream = g_unix_output_stream_new (tmpf.fd, FALSE); + gsize bytes_written; + + /* Generate a boot.mount unit that bind-mounts /sysroot/boot to /boot. + * + * We use DefaultDependencies=no for the same reasons as var.mount: + * to avoid implicit device ordering that can stall after soft-reboot. + * Since this is a bind mount from /sysroot, we only need sysroot.mount. + */ + if (!g_output_stream_printf (outstream, &bytes_written, cancellable, error, + "##\n# Automatically generated by ostree-system-generator\n" + "# Bind mount /boot from /sysroot/boot (same partition)\n##\n\n" + "[Unit]\n" + "Documentation=man:ostree(1)\n" + "DefaultDependencies=no\n" + "After=local-fs-pre.target sysroot.mount\n" + "Before=local-fs.target\n" + "\n" + "[Mount]\n" + "Where=%s\n" + "What=/sysroot/boot\n" + "Options=bind\n", + boot_path)) + return FALSE; + if (!g_output_stream_flush (outstream, cancellable, error)) + return FALSE; + g_clear_object (&outstream); + if (!glnx_fchmod (tmpf.fd, 0644, error)) + return FALSE; + if (!glnx_link_tmpfile_at (&tmpf, GLNX_LINK_TMPFILE_NOREPLACE, normal_dir_dfd, "boot.mount", + error)) + return FALSE; + + /* Ensure it's pulled in by local-fs.target */ + if (!glnx_shutil_mkdir_p_at (normal_dir_dfd, "local-fs.target.requires", 0755, cancellable, + error)) + return FALSE; + if (symlinkat ("../boot.mount", normal_dir_dfd, "local-fs.target.requires/boot.mount") < 0) + return glnx_throw_errno_prefix (error, "symlinkat"); + + return TRUE; +} + /* Generate var.mount */ static gboolean fstab_generator (const char *ostree_target, const bool is_aboot, const char *normal_dir, @@ -237,12 +408,33 @@ fstab_generator (const char *ostree_target, const bool is_aboot, const char *nor * Documentation/filesystems/sharedsubtree.txt and * https://github.com/ostreedev/ostree/issues/2086. This also happens in * ostree-prepare-root.c for the INITRAMFS_MOUNT_VAR case. + * + * We use DefaultDependencies=no to avoid implicit ordering dependencies that + * can cause the mount to stall after a bare `systemctl soft-reboot`. Without + * this, systemd may add implicit After= dependencies on device units, which + * can get stuck in 'tentative' state while udev restarts after soft-reboot. + * Since this is a bind mount from /sysroot (which survives soft-reboot), we + * only need to wait for sysroot.mount and local-fs-pre.target. + * + * Note: We intentionally do NOT add Conflicts=umount.target or + * Before=umount.target here. Adding those creates a dependency deadlock + * with ostree-remount.service during soft-reboot shutdown, because: + * - ostree-remount.service has After=var.mount and Before=local-fs.target + * - Adding Conflicts=umount.target to var.mount creates circular ordering + * where umount.target waits for ostree-remount.service, which waits for + * local-fs.target, which waits for var.mount + * Since /var survives soft-reboot (as a bind mount from /sysroot), systemd + * catches it up from mountinfo and it doesn't need special unmount handling. + * + * See https://issues.redhat.com/browse/RHEL-154075 */ if (!g_output_stream_printf (outstream, &bytes_written, cancellable, error, "##\n# Automatically generated by ostree-system-generator\n##\n\n" "[Unit]\n" "Documentation=man:ostree(1)\n" + "DefaultDependencies=no\n" "ConditionKernelCommandLine=!systemd.volatile\n" + "After=local-fs-pre.target sysroot.mount\n" "Before=local-fs.target\n" "\n" "[Mount]\n" @@ -320,6 +512,10 @@ _ostree_impl_system_generator (const char *normal_dir, const char *early_dir, co if (!require_internal_units (normal_dir, early_dir, late_dir, error)) return FALSE; + if (!sysroot_mount_generator (normal_dir, error)) + return FALSE; + if (!boot_mount_generator (normal_dir, error)) + return FALSE; if (!fstab_generator (ostree_target, is_aboot, normal_dir, early_dir, late_dir, error)) return FALSE; diff --git a/src/switchroot/ostree-prepare-root-static.c b/src/switchroot/ostree-prepare-root-static.c index f8a7a149..2ff202fd 100644 --- a/src/switchroot/ostree-prepare-root-static.c +++ b/src/switchroot/ostree-prepare-root-static.c @@ -237,7 +237,12 @@ main (int argc, char *argv[]) /* Prepare /boot. * If /boot is on the same partition, use a bind mount to make it visible - * at /boot inside the deployment. */ + * at /boot inside the deployment. + * + * Note: The composefs/systemd path (ostree-prepare-root.c) no longer does this - + * it's handled by ostree-system-generator's boot.mount unit instead, which + * supports soft-reboot. But this static path is used without systemd, so the + * generator doesn't run and we must still do it here. */ if (snprintf (srcpath, sizeof (srcpath), "%s/boot/loader", root_mountpoint) < 0) err (EXIT_FAILURE, "failed to assemble /boot/loader path"); if (lstat (srcpath, &stbuf) == 0 && S_ISLNK (stbuf.st_mode)) diff --git a/src/switchroot/ostree-prepare-root.c b/src/switchroot/ostree-prepare-root.c index 97880059..ef5b0044 100644 --- a/src/switchroot/ostree-prepare-root.c +++ b/src/switchroot/ostree-prepare-root.c @@ -280,8 +280,9 @@ main (int argc, char *argv[]) g_variant_builder_add (&metadata_builder, "{sv}", OTCORE_RUN_BOOTED_KEY_SYSROOT_RO, g_variant_new_boolean (sysroot_readonly)); - if (!otcore_mount_boot (root_mountpoint, TMP_SYSROOT, &error)) - errx (EXIT_FAILURE, "%s", error->message); + /* /boot is handled by ostree-system-generator which generates a boot.mount + * unit when /boot is on the same partition. This works for all boot scenarios + * including soft-reboot. See ostree-impl-system-generator.c */ /* Prepare /etc. * No action required if sysroot is writable. Otherwise, a bind-mount for diff --git a/tests-unit-container/test-prepare-root.sh b/tests-unit-container/test-prepare-root.sh index c4f95c1c..012659c0 100755 --- a/tests-unit-container/test-prepare-root.sh +++ b/tests-unit-container/test-prepare-root.sh @@ -56,7 +56,7 @@ done # etc is not transient by default etc_options=$(findmnt -no OPTIONS /target-sysroot/etc) [[ ! $etc_options =~ "upperdir=/run/ostree/transient-etc" ]] -# We don't have /boot as a bind mount by default here +# /boot is handled by the generator, not prepare-root if mountpoint /target-sysroot/boot &>/dev/null; then exit 1 fi @@ -102,16 +102,21 @@ cleanup echo "ok verified etc.transient" -# Set up boot/loader via traditional ostree swapped symlink pattern -# which will cause prepare-root to also make a bind mount. +# Set up boot/loader via traditional ostree swapped symlink pattern. +# /boot bind-mounting has been moved from ostree-prepare-root to +# ostree-system-generator (boot.mount), so prepare-root should NOT +# mount /boot anymore. See ostree-impl-system-generator.c. mkdir /target-sysroot/boot/loader.0 -ln -s /target-sysroot/boot/loader.0 /target-sysroot/boot/loader +ln -s /target-sysroot/boot/loader.0 /target-sysroot/boot/loader mount --bind /target-sysroot /target-sysroot /usr/lib/ostree/ostree-prepare-root /target-sysroot -mountpoint /target-sysroot/boot +if mountpoint /target-sysroot/boot &>/dev/null; then + echo "error: /boot should not be a mountpoint after prepare-root" >&2 + exit 1 +fi cleanup -echo "ok verified /boot" +echo "ok verified /boot is not mounted by prepare-root" diff --git a/tests/kolainst/destructive/soft-reboot.sh b/tests/kolainst/destructive/soft-reboot.sh index f7b26b90..1b883d52 100755 --- a/tests/kolainst/destructive/soft-reboot.sh +++ b/tests/kolainst/destructive/soft-reboot.sh @@ -1,7 +1,8 @@ #!/bin/bash +# shellcheck disable=SC2154 # host_commit is defined in libinsttest.sh set -xeuo pipefail -. ${KOLA_EXT_DATA}/libinsttest.sh +. "${KOLA_EXT_DATA}"/libinsttest.sh prepare_tmpdir @@ -18,7 +19,7 @@ assert_jq findmnt.json '.filesystems[0].options | contains("ro")' require_writable_sysroot assert_soft_reboot_count() { - assert_streq $(systemctl show -P SoftRebootsCount) $1 + assert_streq "$(systemctl show -P SoftRebootsCount)" "$1" } case "${AUTOPKGTEST_REBOOT_MARK:-}" in @@ -27,14 +28,46 @@ case "${AUTOPKGTEST_REBOOT_MARK:-}" in systemctl mask --now zincati assert_soft_reboot_count 0 + + # First, test a bare systemctl soft-reboot (without ostree's prepare-soft-reboot). + # This tests the fix for https://issues.redhat.com/browse/RHEL-154075 where + # a bare soft-reboot would cause /var to fail to mount due to the generated + # var.mount unit getting stuck waiting on device units. + echo "Testing bare systemctl soft-reboot (no /run/nextroot)..." + # Verify /run/nextroot is not set up + test '!' -d /run/nextroot || ! mountpoint -q /run/nextroot + /tmp/autopkgtest-soft-reboot-prepare "bare-soft-reboot" + systemctl soft-reboot + ;; + "bare-soft-reboot") + # After bare soft-reboot, verify we're still running the same deployment + # and critically, that /var is mounted and the system is healthy. + echo "Verifying post-bare-soft-reboot state..." + assert_soft_reboot_count 1 + + # The key assertion: /var must be mounted for the system to be functional + mountpoint /var + # Verify /var is actually usable (we can write to it) + touch /var/tmp/soft-reboot-test-marker + rm /var/tmp/soft-reboot-test-marker + + # /boot must also be mounted (handled by generator's boot.mount) + mountpoint /boot + + # We should still be on the same deployment (no ostree-level change) + assert_status_jq '.deployments[0].booted' + + echo "ok bare soft-reboot" + + # Now continue with the rest of the soft-reboot tests assert_status_jq '.deployments[0].pending | not' '.deployments[0].["soft-reboot-target"] | not' # Create a synthetic commit for upgrade cd /ostree/repo/tmp - ostree checkout -H ${host_commit} t + ostree checkout -H "${host_commit}" t unshare -m /bin/sh -c 'mount -o remount,rw /sysroot && cd /ostree/repo/tmp/t && touch usr/etc/new-file-for-soft-reboot usr/share/test-file-for-soft-reboot' ostree commit --no-bindings --parent="${host_commit}" -b soft-reboot-test -I --consume t - newcommit=$(ostree rev-parse soft-reboot-test) + ostree rev-parse soft-reboot-test >/dev/null # Deploy the new commit normally first ostree admin deploy --stage soft-reboot-test @@ -59,12 +92,12 @@ case "${AUTOPKGTEST_REBOOT_MARK:-}" in "2") # After soft reboot, verify we're running the new deployment echo "Verifying post-soft-reboot state..." - assert_soft_reboot_count 1 + assert_soft_reboot_count 2 expected_commit=$(ostree rev-parse soft-reboot-test) if [ "${host_commit}" != "${expected_commit}" ]; then - echo "ERROR: Expected commit ${host_commit}, but got ${current_commit}" + echo "ERROR: Expected commit ${expected_commit}, but got ${host_commit}" exit 1 fi @@ -85,7 +118,7 @@ case "${AUTOPKGTEST_REBOOT_MARK:-}" in ostree admin prepare-soft-reboot --reboot 1 ;; "3") - assert_soft_reboot_count 2 + assert_soft_reboot_count 3 # Only from the first updated target test '!' -f /etc/new-file-for-soft-reboot @@ -112,10 +145,10 @@ case "${AUTOPKGTEST_REBOOT_MARK:-}" in # Now, test the intersection of staged deployments and soft rebooting # Create another synthetic commit cd /ostree/repo/tmp - ostree checkout -H ${host_commit} t + ostree checkout -H "${host_commit}" t unshare -m /bin/sh -c 'mount -o remount,rw /sysroot && cd /ostree/repo/tmp/t && touch usr/share/test-staged-2-for-soft-reboot' ostree commit --no-bindings --parent="${host_commit}" -b soft-reboot-test-staged-2 -I --consume t - newcommit=$(ostree rev-parse soft-reboot-test-staged-2) + ostree rev-parse soft-reboot-test-staged-2 >/dev/null ostree admin deploy --stage soft-reboot-test-staged-2 assert_status_jq '.deployments[0].staged' '.deployments[0].["soft-reboot-target"] | not' \ @@ -134,7 +167,7 @@ case "${AUTOPKGTEST_REBOOT_MARK:-}" in systemctl reboot ;; "4") - assert_soft_reboot_count 3 + assert_soft_reboot_count 4 # Completion of soft reboot into non-staged assert_status_jq '.deployments[0].booted' '.deployments[0].["soft-reboot-target"] | not' \ '.deployments[1].booted | not' '.deployments[1].["soft-reboot-target"] | not'